# codeing:utf-8
# -!- coding: utf-8 -!-
import datetime
from conf import *
import pymysql
import os
import AnalyzeTableMain1 as AnalyzeTableMain
import AnalyzeTextMain1 as AnalyzeTextMain
import difflib
from pandas import DataFrame
import re
from loguru import logger
import zipfile
import json
import hashlib
import shutil
from conf1 import Conf1
import time
from multiprocessing import Process,Queue
from common import *

class xm_to_id():
    def __init__(self):
        self.html = ""

    def get_ToChinaText(self, value):
        v = [['[Ⅰ一１]', '1'], ['[Ⅱ二２]', '2'], ['[Ⅲ三３]', '3'], ['[Ⅳ四４]', '4'], ['[Ⅴ五５]', '5'],
             ['[Ⅵ六６]', '6'], ['[Ⅶ七７]', '7'], ['[Ⅷ八８]', '8'], ['[Ⅸ九９]', '9'], ['十', '10'],
             ['[ａＡ]', 'a'], ['[ｂＢ]', 'b'], ['[ｃＣ]', 'c'], ['[ｄＤ]', 'd'], ['[ｅＥ]', 'e'], ['[ｆＦ]', 'f'], ['[ｇＧ]', 'g'],
             ['[ｈＨ]', 'h'],
             ['[ｉＩ]', 'i'], ['[ｊＪ]', 'j'], ['[ｋＫ]', 'k'], ['[ｌＬ]', 'l'], ['[ｍＭ]', 'm'], ['[ｎＮ]', 'n'], ['[ｏＯ]', 'o'],
             ['[ｐＰ]', 'p'],
             ['[ｑＱ]', 'q'], ['[ｒＲ]', 'r'], ['[ｓＳ]', 's'], ['[ｔＴ]', 't'], ['[ｕＵ]', 'u'], ['[ｖＶ]', 'v'], ['[ｗＷ]', 'w'],
             ['[ｘＸ]', 'x'],
             ['[ｙＹ]', 'y'], ['[ｚＺ]', 'z']]
        for i in v:
            value = re.sub(i[0], i[1], value)
        return value

    def removePunctuation(self, value):
        return re.sub(r'(－|,|，|\.|。|/|•|·|、|\||#|;|；|''|‘|’|“|”|《|》|\<|\>|:|：|？|\?|\[|\]|【|】|\{|\}|｛|｝|_|~|!|！|-|`|@|\$|￥|%|…|\^|&|\*|"|\=|\+|\(|\)|（|）|——|—|～)', "", value, flags=re.S | re.I)

    def get_str_sort(self, value):
        v = list(set(re.sub(r'\s+', '', value, flags=re.S | re.I)))
        v.sort()
        return "".join(v)

    def get_id(self, value):
        v = self.get_ToChinaText(value)
        v = re.sub(r'总价|承包|设计施工一体化|采购|监理|EPC|工程管理|工程咨询|项目管理|项目|施工|勘察|设计|招标|代理|合同验收|标段|第|联合体|总承包|专业承包|标|单位|建设|不分标段|投资', '', v, flags=re.S | re.I)
        v = re.sub(r'([^\d])20\d{6}([^\d])', '\1\2', v, flags=re.S | re.I)
        return self.get_str_sort(self.removePunctuation(v))

    def get_name(self, value):
        return re.sub(r'磋商|性|全电子|联合体|网上|竞价|关于|变更|正常|及|的|[开中招评投]标|结果|公[示告布开]|通知|投资人|成交|候选(人|单位)|表|复核|重新|拟|标准|文本|选择|一览|记录|情况|报告|书|第|[0-9一二三四五六七八九十]+次|[复重]评|项目|代码|资格|预审|竞争|性|谈判|\d{4}年\d{2}月\d{2}日|验收|结果|联合体(\d+)?', '', self.removePunctuation(value).upper(), flags=re.S | re.I)

class Producer(Process):
    def __init__(self,name_queue,date_start,date_end,data_source_id_jugde):
        super().__init__()
        self.name_queue = name_queue
        self.date_start = date_start
        self.date_end = date_end
        self.data_source_id_jugde = data_source_id_jugde
        self.father_path_yesanalyze = 'E:\\project\\DataBaseData\\YesAnalyze'

    def run(self):
        type_dir = ['3']
        if os.path.exists('yeszip.csv'):
            not_ls = list(pd.read_csv('yeszip.csv'))
        else:
            not_ls = []
        logger.info('----------having a part----------')
        for typee in type_dir:
            date_dir = os.listdir(os.path.join(self.father_path_yesanalyze, typee))
            for data_source_id in date_dir:
                data_source_id_dir = os.listdir(os.path.join(self.father_path_yesanalyze, typee, data_source_id))
                for date in data_source_id_dir:
                    if date >= self.date_end or date < self.date_start:
                        continue
                    if data_source_id != self.data_source_id_jugde and self.data_source_id_jugde != 'All':
                        continue
                    zip_dir = os.listdir(os.path.join(self.father_path_yesanalyze, typee, data_source_id, date))
                    for zipp in zip_dir:
                        if zipp not in not_ls:
                            self.name_queue.put([typee, data_source_id, date, zipp])
                self.name_queue.put(['data_source_id_end', typee, data_source_id])

class Analyze(Process):


    def __init__(self,file_list):
        super().__init__()
        self.zippnum = 100
        self.file_list = file_list
        self.father_analyzedata = 'E:\\project\\DataBaseData\\OldAnalyze\\Old'
        self.father_noanalyze = 'E:\\project\\DataBaseData\\YesAnalyze'
        self.father_path = 'E:\\project\\DataBaseData'


    def wrong_analyze_insert(self,conn,cursor,id_not_change):
        sql = 'insert into analyze_wrong_data(zip_name,identify,wrong_type,date_flag) values(%s,%s,%s,%s)'
        try:
            conn.ping(reconnect=True)
            cursor.executemany(sql, id_not_change)
            conn.commit()
        except:
            DataFrame([id_not_change]).to_csv('wrong_analyze.csv', header=None, index=None, mode='a')


    def write_io(self,typee,data_source_id,end_result,identify):
        try:
            dir_now = self.father_analyzedata + str(typee)
            if not os.path.exists(os.path.join(dir_now,str(data_source_id))):
                os.mkdir(os.path.join(dir_now,str(data_source_id)))
            namee = os.path.join(dir_now,str(data_source_id),datetime.datetime.strftime(datetime.datetime.now(),'%Y%m%d%H%M%S')+'_'+identify+'.json')
            with open(namee,'w',encoding = 'utf-8') as f:
                f.write(end_result)
            return True,namee
        except Exception as e:
            logger.debug('write wrong '+str(e))
            return False,namee


    def zipe(self,typee,key = False):
        data_ls = os.listdir(os.path.join(self.father_analyzedata+str(typee)))
        for zip_lsi in data_ls:
            if '.zip' in zip_lsi or 'back_up' in zip_lsi:
                continue
            json_data = os.listdir(os.path.join(self.father_analyzedata+str(typee),zip_lsi))
            if len(json_data) >= self.zippnum or key:
                shutil.copyfile(os.path.join(self.father_path, 'TaskEnd'),os.path.join(self.father_analyzedata + str(typee), zip_lsi, 'TaskEnd'))
                try:
                    zip_1 = zipfile.ZipFile(os.path.join(self.father_analyzedata+str(typee),f'{zip_lsi}_{datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d%H%M%S")}.zip'),'w')
                    for data in os.listdir(os.path.join(self.father_analyzedata+str(typee),zip_lsi)):
                        zip_1.write(os.path.join(self.father_analyzedata+str(typee), zip_lsi, data),data,zipfile.ZIP_DEFLATED)
                    zip_1.close()
                    if os.path.exists(os.path.join(self.father_analyzedata+str(typee), zip_lsi)):
                        shutil.rmtree(os.path.join(self.father_analyzedata+str(typee), zip_lsi))
                except Exception as e:
                    logger.debug('zip wrong '+str(e))

    def concat(self,end_result, typee):
        out_dict = end_result['表格']
        if typee == '3':
            yeji_ls_concat = ['第一中标候选人', '第二中标候选人', '第三中标候选人']
            for yeji_ls_concati in yeji_ls_concat:
                out_dict[yeji_ls_concati] = out_dict[yeji_ls_concati] + end_result['文本'][yeji_ls_concati]
            for yeji_ls_concati in yeji_ls_concat:
                company_dict = {}
                out_result = []
                for i in out_dict[yeji_ls_concati]:
                    if i['项目负责人_资格_编号'] in ['_', '__']:
                        i['项目负责人_资格_编号'] = ''
                    key = i['公司名称']
                    company_dict[key] = company_dict.get(key, []) + [i]
                for i in company_dict:
                    biao_dict = {}
                    for j in company_dict[i]:
                        key = j['标段']
                        biao_dict[key] = biao_dict.get(key, []) + [j]
                    for j in biao_dict:
                        if len(biao_dict[j]) == 1:
                            out_result.append(biao_dict[j][0])
                        else:
                            biao_out = biao_dict[j][0]
                            for m in biao_dict[j][1:]:
                                try:
                                    money = float(m['中标金额'])
                                    if money > biao_out['中标金额']:
                                        biao_out = m
                                except:
                                    if biao_out['中标金额'] == 0 or biao_out['中标金额'] == '' or biao_out['中标金额'] == '/':
                                        biao_out = m
                            out_result.append(biao_out)
                out_dict[yeji_ls_concati] = out_result

        for key in out_dict:
            if re.search('^第[一二三]中标候选人$', key):
                continue

            tt1 = out_dict[key]
            if isinstance(tt1, float):
                try:
                    tt1 = str(tt1)
                except:
                    pass
            tt2 = end_result['文本'][key]
            if isinstance(tt2, float):
                try:
                    tt2 = str(tt2)
                except:
                    pass

            if len(tt1) == 0 and len(tt2) != 0:
                out_dict[key] = end_result['文本'][key]

        return out_dict

    def handle(self,**kwargs):
        conn = kwargs['conn']
        cursor = kwargs['cursor']
        params = kwargs['params']
        config = kwargs['config']
        config1 = kwargs['config1']
        json_data = kwargs['json_data']
        date = kwargs['date']
        typee= kwargs['typee']
        zipp = kwargs['zipp']
        json_data_be = kwargs['json_data_be']
        id_not_change = []
        data_type = json_data_be['data_type']
        data_source_id = json_data_be['data_source_id']
        params['data_source_id'] = data_source_id
        title = json_data_be['title']
        title_date = json_data_be['titleDate']
        url = json_data_be['url']
        html = json_data_be['html_content']
        params['html'] = html
        params['type'] = typee
        identify = hashlib.md5((title + url + title_date).encode('utf-8')).hexdigest()
        table_name = json_data_be['table_name']
        if data_type in [1,4]:
            html_use = html
        else:
            html_use = ''
        type_sec = config1['type_sec'][typee]
        end_result = {"项目名称采集": title, "中标日期": str(title_date), "提取数据": {}, "网页链接": url,"任务ID": data_source_id,'任务表':table_name,'type':data_type,'html':html_use}
        if typee != '3':
            end_result["提取数据"] = {key: '' if key != '投标要求' else [] for key  in list(config1[type_sec].keys())}
        else:
            end_result["提取数据"] = {key: '' for key in list(config1['achivement_sec_common'].keys())}
            end_result["提取数据"]['第一中标候选人'] = []
            end_result["提取数据"]['第二中标候选人'] = []
            end_result["提取数据"]['第三中标候选人'] = []

        end_result_tt = {'表格':{},'文本':{}}
        try:
            read = AnalyzeTableMain.AnalyzeTable(**params)
            end_dict = read.main()
        except Exception as e:
            end_dict = end_result["提取数据"]
            id_not_change.append((table_name,identify,0,date))
            logger.info(e)
            try:
                DataFrame([[e,url,data_source_id]]).to_csv('wrong_table.csv',header = None,index = None,mode = 'a')
            except Exception as e:
                logger.info('import worng '+ str(e))

        end_result_tt['表格'] = end_dict
        try:
            read = AnalyzeTextMain.AnalyzeText(**params)
            end_dict = read.main()
        except Exception as e:
            end_dict = end_result["提取数据"]
            id_not_change.append((table_name, identify,1,date))
            logger.info(str(e))
            try:
                DataFrame([[e,url,data_source_id]]).to_csv('wrong_text.csv',header = None,index = None,mode = 'a')
            except Exception as e:
                logger.info('import worng '+ str(e))
        end_result_tt['文本'] = end_dict
        end_result["提取数据"] = self.concat(end_result_tt,typee)
        company_ls = ['第一中标候选人', '第三中标候选人', '第三中标候选人']
        if typee == '3':
            xid = xm_to_id()
            xm_info_ = {"项目标识": "", "项目名称": xid.get_name(end_result['项目名称采集']),
                        '公司名称': '',
                        "标段": "", "中标金额": "", "项目负责人": "",
                        "项目分类": ""}
            company_handle = {}
            for j in company_ls:
                if end_result["提取数据"][j] == []:
                    company_handle[j] = []
                else:
                    for i in end_result["提取数据"][j]:
                        v = re.sub('(司|院|局|队|中心|所)[与和]', r'\1;', i["公司名称"])
                        v = re.sub(
                            "(（?(牵头人|成员|联合体)）?|;|：|◄|、|(设计|施工|监理|勘察)方：?|/|&|名称|第[0-9一二三四五六七八九十]+包|包件[0-9一二三四五六七八九十]+|，|,|[0-9一二三四五六七八九十]+包)|？|；|第一中标(供应商|候选人)|-|／|－|（包\d+）|（包[一二三四五六七八九十]+）|（\d+）|（[a-z](片区|包)）|（包[a-z]）|([中投]标|成交)(报?价|金额|单位|企业|人)为?|\d+(\.\d+)?万?元|(成交)?报价|拟?确定|最低的?",
                            ":", v, flags=re.S | re.I)
                        v = re.sub(u"\u3000", u"", v, flags=re.S | re.I)
                        line = re.split(":", v)
                        for k in line:
                            xm_info = xm_info_.copy()
                            val_obj = re.search(config['company_chose'][0], k, re.S | re.I)
                            if (k != "") and (len(k) >= 5) and val_obj:
                                if k[0] == '（':
                                    k = k[1:]
                                if (k[-1] == '为') or (k[-1] == '）') or (k[-1] == '(') and (
                                        '(' not in k):
                                    k = k[:-1]
                                xm_info["公司名称"] = k
                                xm_info["标段"] = i["标段"]
                                xm_info["中标金额"] = i["中标金额"]
                                if len(i["项目负责人"]) >= 2:
                                    value = i["项目负责人"]
                                    for r in config['replace_manage_name']:
                                        value = re.sub(r[0], r[1], value, flags=re.S | re.I)
                                    dd = []
                                    d = re.split(":", value)
                                    for di in d:
                                        if di in ['施工', '设计', '勘察', '监理']:
                                            continue
                                        if (len(di) >= 2) and (len(di) <= 4):
                                            obj = re.match(config['human_chose'][0], di,
                                                           re.S | re.I)
                                            if obj:
                                                dd.append(di)
                                    if len(dd) > 0:
                                        xm_info["项目负责人"] = dd[0]
                                xm_info["项目名称"] = xid.get_name(end_result['项目名称采集'])
                                xm_info["项目标识"] = xid.get_id(xm_info["项目名称"])
                                company_handle[j] = company_handle.get(j, []) + [xm_info]
            for company_lsi in company_ls:
                if company_lsi in company_handle:
                    end_result["提取数据"][company_lsi] = company_handle[company_lsi]
        end_result = json.dumps(end_result, ensure_ascii=False)
        if id_not_change != []:
            self.wrong_analyze_insert(conn,cursor,id_not_change)
        id_not_change = []
        key,namee = self.write_io(typee,data_source_id,end_result,identify)
        if not key:
            if os.path.exists(namee):
                os.remove(namee)
            DataFrame([[typee,data_source_id,date, zipp, json_data]]).to_csv('writeio_wrong.csv', header=None, index=None, mode='a')


    def run(self):
        while True:
            try:
                conn = pymysql.connect(
                    host='192.168.0.160',
                    port=3306,
                    user='root',
                    passwd='lubanjianye',
                    charset='utf8',
                    database='data_collection_server',
                    cursorclass=pymysql.cursors.DictCursor)
                cursor = conn.cursor()
                args = {'conn': conn,
                        'cursor': cursor}
                conf = Conf(**args)
                config = conf.main()
                conf1 = Conf1(**args)
                config1 = conf1.main()
                confunc = ConFunc(config)
                rubbish_word = confunc.rubbish_word
                replace_word = confunc.replace_word
                get_money = confunc.get_money
                clear_solo = confunc.clear_solo
                params = {
                    'html': 'C:/Users/Administrator/Desktop/table_easy/table30.html',
                    'config': config,
                    'rubbish_word': rubbish_word,
                    'replace_word': replace_word,
                    'get_money': get_money,
                    'database': True,
                    'data_source_id': '',
                    'clear_solo': clear_solo,
                    'config1':config1
                }
                break
            except Exception as e:
                logger.debug('pre wrong '+str(e))
                continue
        handle_params = {
            'conn': conn,
            'cursor': cursor,
            'params': params,
            'config': config,
            'date': '',
            'typee': '',
            'zipp': '',
            'json_data_be': '',
            'json_data': '',
            'config1':config1
        }
        while not self.file_list.empty():
            all_use_data = self.file_list.get()
            if all_use_data[0] == 'data_source_id_end':
                self.zipe(all_use_data[1],True)
                continue
            typee = all_use_data[0]
            data_source_id = all_use_data[1]
            date = all_use_data[2]
            zipp = all_use_data[3]
            logger.info(f'----------{zipp} start----------')
            if not os.path.exists(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp)):
                continue
            if typee == '3':
                sql = f'select data_extra from id_to_name_handle where data_source_id = {data_source_id}'
                cursor.execute(sql)
                data_extra = cursor.fetchone()
                if data_extra != None:
                    if data_extra['data_extra'] != None:
                        data_extra = data_extra['data_extra']
                        if data_extra == 0:
                            continue
            if typee == '2':
                continue
            try:
                z = zipfile.ZipFile(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp))
            except:
                logger.debug('not a zip {0}'.format(os.path.join(self.father_noanalyze,typee,data_source_id,date,zipp)))
                continue
            json_name = z.namelist()

            for json_data in json_name:
                if '任务' in json_data:
                    continue
                try:
                    json_data_be = json.loads(z.read(json_data).decode('utf-8'))
                except Exception as e:
                    logger.debug('loads wrong '+str(e))
                    DataFrame([[typee,data_source_id,date,zipp,json_data]]).to_csv('loads_wrong.csv', header=None, index=None, mode='a')
                    continue
                handle_params['date'] = date
                handle_params['typee'] = typee
                handle_params['zipp'] = zipp
                handle_params['json_data_be'] = json_data_be
                handle_params['json_data'] = json_data
                try:
                    self.handle(**handle_params)
                except Exception as e:
                    logger.debug('handle wrong ' + str(e))
                    DataFrame([[typee, data_source_id,date, zipp, json_data]]).to_csv('handle_wrong.csv', header=None,index=None, mode='a')
                    continue
            z.close()
            self.zipe(typee,False)
            DataFrame([[zipp]]).to_csv('yeszip.csv', index=None, header=None, mode='a')
            handle_params['json_data_be'] = ''


def main():
    name_queue = Queue()
    for i in range(1):
        t = Producer(name_queue,'2021-01-01','2021-02-01','All')
        t.start()
    time.sleep(5)
    for i in range(1):
        t = Analyze(name_queue)
        t.start()

if __name__ == '__main__':
    main()

